{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "import math\n",
    "import operator"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "\"\"\"Create a data sample\n",
    "    dataset-Experiment sample cut entry\n",
    "    classVec-Category label vector\n",
    "\"\"\""
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "def loadDataSet():\n",
    "    dataset = [['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],\n",
    "               ['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],\n",
    "               ['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],\n",
    "               ['stop', 'posting', 'stupid', 'worthless', 'garbage'],\n",
    "               ['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],\n",
    "               ['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]\n",
    "    classVec = [0, 1, 0, 1, 0, 1]  # 1 good,0 not good\n",
    "    return dataset, classVec"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "\"\"\"tfidf algorithm\n",
    "\"\"\""
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "def feature_select(list_words):\n",
    "    #Total word frequency statistics\n",
    "    doc_frequency=defaultdict(int)\n",
    "    for word_list in list_words:\n",
    "        for i in word_list:\n",
    "            doc_frequency[i]+=1\n",
    "    #tf value\n",
    "    word_tf={}\n",
    "    for i in doc_frequency:\n",
    "        word_tf[i]=doc_frequency[i]/sum(doc_frequency.values())\n",
    "\n",
    "    #idf value\n",
    "    doc_num=len(list_words)\n",
    "    word_idf={}\n",
    "    word_doc=defaultdict(int)#Store the number of documents containing the word\n",
    "    for i in doc_frequency:\n",
    "        for j in list_words:\n",
    "            if i in j:\n",
    "                word_doc[i]+=1\n",
    "    for i in doc_frequency:\n",
    "        word_idf[i]=math.log(doc_num/(word_doc[i]+1))\n",
    "\n",
    "    #tf*idf\n",
    "    word_tf_idf={}\n",
    "    for i in doc_frequency:\n",
    "        word_tf_idf[i]=word_tf[i]*word_idf[i]\n",
    "\n",
    "    #For the dictionary, the value is sorted by large to small\n",
    "    dict_feature_select = sorted(word_tf_idf.items(), key=operator.itemgetter(1), reverse=True)\n",
    "    return dict_feature_select"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('to', 0.0322394037469742), ('stop', 0.0322394037469742), ('worthless', 0.0322394037469742), ('my', 0.028288263356383563), ('dog', 0.028288263356383563), ('him', 0.028288263356383563), ('stupid', 0.028288263356383563), ('has', 0.025549122992281622), ('flea', 0.025549122992281622), ('problems', 0.025549122992281622), ('help', 0.025549122992281622), ('please', 0.025549122992281622), ('maybe', 0.025549122992281622), ('not', 0.025549122992281622), ('take', 0.025549122992281622), ('park', 0.025549122992281622), ('dalmation', 0.025549122992281622), ('is', 0.025549122992281622), ('so', 0.025549122992281622), ('cute', 0.025549122992281622), ('I', 0.025549122992281622), ('love', 0.025549122992281622), ('posting', 0.025549122992281622), ('garbage', 0.025549122992281622), ('mr', 0.025549122992281622), ('licks', 0.025549122992281622), ('ate', 0.025549122992281622), ('steak', 0.025549122992281622), ('how', 0.025549122992281622), ('quit', 0.025549122992281622), ('buying', 0.025549122992281622), ('food', 0.025549122992281622)]\n",
      "32\n"
     ]
    }
   ],
   "source": [
    "if __name__=='__main__':\n",
    "    data_list,label_list=loadDataSet()\n",
    "    features=feature_select(data_list)\n",
    "    print(features)\n",
    "    print(len(features))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}